import requests
from bs4 import BeautifulSoup


def get_novel_chapters():
    data = []
    url = 'https://www.bqg220.com/biquge/89523/'

    r = requests.get(url)
    r.encoding = 'utf-8'
    soup = BeautifulSoup(r.text, 'html.parser')
    for dd in soup.find_all('dd'):
        link = dd.find('a')
        if not link:
            continue
        elif link['href'].startswith('/biquge/89523'):
            data.append(['https://www.bqg220.com' + link['href'], link.text])
    return data


def get_chapter_content(url):
    r = requests.get(url)
    r.encoding = 'utf-8'
    soup = BeautifulSoup(r.text, 'html.parser')
    if soup.find('div', id='chaptercontent').get_text().find('/p') != -1:
        content = '    ' + soup.find('div', id='chaptercontent').get_text().replace('　　p', '').replace('/p', '\n    ').replace(
            '    \n    　　请收藏本站：https://www.bqg220.com。笔趣阁手机版：https://m.bqg220.com \n『点此报错』『加入书签』', '\n')
    else:
        content = '    ' + soup.find('div', id='chaptercontent').get_text().strip().replace('　　', '\n    ').replace(
            '    请收藏本站：https://www.bqg220.com。笔趣阁手机版：https://m.bqg220.com \n『点此报错』『加入书签』', '\n')
    return content


f = open('novel/我不过是个大罗金仙.txt', 'a', encoding='utf-8')
for chapter in get_novel_chapters():
    print(f'开始爬取 {chapter[1]}')
    f.write(chapter[1] + '\n\n')
    f.write(get_chapter_content(chapter[0]))
f.close()

print('爬取完成')
